const express = require('express'),
  router = express.Router(),
  util = require("../util/util"),
  spider = require("../model/spider"),
  chaptermodel = require("../model/chapter"),
   Sequelize = require('sequelize'),
  reObj = util.reObj,
   puppeteer = require('puppeteer'),
   fs = require('fs'),
   path = require('path'),
   cheerio = require('cheerio'),
   superagent = require('superagent'),
   async = require("async");

  function sortId(a,b){
    return a.num-b.num
}
async function getChapter(url,num){
  console.log("url:"+url)
  console.log("page:"+num)
    const browser = await puppeteer.launch({headless: true,args: ['--no-sandbox','--ignore-certificate-errors']})
    const page = await browser.newPage()
    const preloadFile = fs.readFileSync(path.resolve(__dirname,"./ele.js"), 'utf8');//注入影子dom破解
    await page.evaluateOnNewDocument(preloadFile);
    //书籍地址
    url=url+"?page=";
    await page.goto(url+num)
  
      let arr=[]
      const html = await page.content();
      let $ = cheerio.load(html);
      console.log("imgsrc:"+$(".img-container img").attr("src"))
      let data={
          name:$(".book-title").text(),//书名
          author:$(".authors .value").text(),//作者
          description:$("#bookinfo p").text().trim(),//描述
          avatar:$(".img-container img").attr("src")
      }
     
      
      let posturl="http://localhost:6208/book/add"
   
      let result=await  superagent.post(posturl).send(data)
      
      console.log("result:"+JSON.stringify(result.text))
      let  bookid=JSON.parse(result.text).data.id
      spider.findOrCreate({
        where: {bookid:bookid},
        defaults: {bookid:bookid,bookname:JSON.parse(result.text).data.name,status:0}
    })
      console.log("bookid:"+bookid)
      $(".chapter-item").each(function () {
          let obj={}
          let num=$(this).find("a").attr("href").split(".")[0].split("/")
          obj.name=$(this).find(".chapter-title").text()
          obj.num=num[num.length-1].slice(1)
          obj.href="https://swnovels.com"+$(this).find("a").attr("href")
          arr.push(obj)
          
      })
      
      let arrSort=arr.sort(sortId)
      let postChapterUrl="http://localhost:6208/chapter/add"
      
  
      async.eachSeries(arrSort, async function (item, callback) {
  
  //添加章节
  
          console.log(JSON.stringify(item))
           let chapterData={
              bookid:bookid,//如果章节爬取不全，需要手动填写bookid
              name:item.name,
              url:item.href          
          }
         // console.log("chapterData:"+JSON.stringify(chapterData))
          let chapterResult=await superagent.post(postChapterUrl).send(chapterData)
       //   console.log("chapterResult:"+JSON.stringify(chapterResult)) 
          //callback(null,"success");
         
      },async function (err,www) {
  
          console.log("completed")
      });
   
  
     await browser.close()
}
async function getChapterContent(item){
  // const browser = await puppeteer.launch({
//   headless: false,
//   executablePath: '/Applications/Google Chrome.app/Contents/MacOS/Google Chrome'
// })
const browser = await puppeteer.launch({headless: true,args: ['--no-sandbox','--ignore-certificate-errors']})
const page = await browser.newPage()

const preloadFile = fs.readFileSync(path.resolve(__dirname,"./ele.js"), 'utf8');//注入影子dom破解
await page.evaluateOnNewDocument(preloadFile);
await page.goto(item.url)


  const html = await page.content();
  let $ = cheerio.load(html);
const textContent = await page.evaluate(() => {
  const list = document.querySelectorAll('.ctp')
  const result = []
  for (let i = 0; i < list.length; i++) {
    const item = list[i]
    if (item && item.shadowRoot) {
      console.log(item)
      result.push(item.shadowRoot.innerHTML)
    } else {
      result.push(null)
    }
  }
  return result
});


  let $$ = cheerio.load(textContent.toString());
 $$("style").remove()
  let str=$("p").toString()+$$("p").toString()

  console.log("content"+str)
  //http://localhost:6208/chapter/update
  let chapterData={
      id:item.id,        
      content:str         
  }
  await superagent.post("http://localhost:6208/chapter/update").send(chapterData)
 await browser.close()
}
/**
 * @function 爬取书籍
 * @author Eric
 * @date 2018-08-01
 * @param name,code,addr,area,status
 */

router.post('/', (req, res) => {
  let num=req.body.page//[1,2,3]
  let bookurl=req.body.url
  let arr=[]  
  for(i=1;i<=num;i++){
    
    arr.push(i)
  }
  
   async.eachSeries(arr, async function (item, callback) {



      console.log("page"+JSON.stringify(item))
      //callback(null,item);
     await getChapter(bookurl,item)
     

     
  },async function (err,www) {
    
      
      console.log("completed page"+www)
      console.log("err"+err)
  }); 
  res.send(reObj(0, '开始爬取，请耐心等候!可在章节里检查',[]))
})

router.post('/writechapter', async (req, res) => {
  let chapterData={
    bookid:req.body.bookid
    /* pagerow:30,
    page:8    */
}


let spiderRecord=await spider.findOne({where: {status:0}})
let chapter=await chaptermodel.findAndCountAll({where: {bookid:spiderRecord.bookid,content:{$eq: null}}})
console.log("chapter:"+JSON.stringify(chapter))
if(chapter.count===0)
{
  spider.update({status:1}, {where: {id:spiderRecord.id}})
  console.log("completed")
  
}
else
{
  
  console.log("not completed")


if(spiderRecord&&spiderRecord.bookid)
{
  chapterData.bookid=spiderRecord.bookid
}
//console.log("chapterDatawwww:"+JSON.stringify(chapterData))
let chapterResult=await superagent.post("http://localhost:6208/chapter/findall").send(chapterData)
//console.log("chapterResult:"+JSON.stringify(chapterResult.text))
let list=JSON.parse(chapterResult.text)
console.log("list length:"+list.data.length)
  async.eachSeries(list.data, async function (item, callback) {
      //查询conent为null，并补全conent内容
      
    if(!item.content)
    {        
        console.log("item:"+JSON.stringify(item))
        await getChapterContent(item) 
    }
     
      
  },async function (err,www) {
    //res.send(reObj(0, '开始爬取，请耐心等候!可在章节里检查',[]))
      console.log("completed page"+www)
      console.log("err"+err)
  }); 
}

res.send(reObj(0, '开始爬取，请耐心等候!可在章节里检查',[]))
})

module.exports = router;
